#!/usr/local/bin/pythonw
# -*- coding: utf-8 -*-
import re
import urllib
import time
import hashlib
import random
import cookielib, urllib2
import sys
import codecs
import os

import glob as glob
import tool_deft as td
from BeautifulSoup import BeautifulSoup,BeautifulStoneSoup,NavigableString,Tag

def count_p(bs) :
  cpt = 0
  corps = bs.doc.article.corps
  l = 0
  for p in corps :
    if type(p) == Tag :
      try :
        l += len(p.string)
      except :
        pass
      cpt += 1
  resume = bs.doc.article.resume
  cpt2 = 0
  for r in resume :
    if type(r) == Tag :
      cpt2 += 1
    
  return cpt,l,cpt2

def split_sentence(s,i) :
  motif = '(.+?(\. ))'
  l = split_generique(s,i,motif)
  return l

def split_virgulot(s,i) :
  motif = '(.+?([,;] ))'
  l = split_generique(s,i,motif)
  return l

def split_word(s,i) :
  motif = '(.+?(\s))'
  l = split_generique(s,i,motif)
  return l

def get_forest(bs) :
  corps = bs.doc.article.corps
  res = []
  for p in corps :
    if type(p) == Tag :
        u = []
#      try :
        s = p.string
        #seg_list = [split_word, split_virgulot, split_sentence]
        seg_list = [split_word,split_virgulot,split_sentence]
        seg_list.reverse()
        i_start= (0,len(s))
        u = rec_split(s, i_start, seg_list, 0)
#        parcourt_aux(s,u,0)
        h = get_singleton_height(u)
        pp = get_max_singleton(u)
        print h,pp
#        if h > 0 :
#          print s
#      except :
#        pass

def parcourt_aux(s,a,p) :
  i,sons = a
  print "   "*p,s[i[0]:i[1]]
  for a_sons in sons :
    parcourt_aux(s,a_sons,p+1)

def get_height(arbre):#hauteur = hauteur fils+1
  if len(arbre[1]) == 0 :
    return 1
  return 1 + max([get_height(fils) for fils in arbre[1]])

def get_singleton_height(arbre):
#  print arbre,len(arbre)
  if len(arbre[1])!=1:
    return 0
  else:
    return 1+get_singleton_height(arbre[1][0])

def locate_all_singleton(arbre):
  return locate_all_singleton_aux(arbre,[],[])

def locate_all_singleton_aux(arbre,l,pos_arbre):
  h = get_singleton_height(arbre)
  if h == 0 :
    for i,son in enumerate(arbre[1]):
      pos_son = pos_arbre+[i]
      l = locate_all_singleton_aux(son,l,pos_son)
    print l
    return l
  l.append((h,pos_arbre[:]))
  next_tree = arbre
  for i in xrange(0,h):
    next_tree = next_tree[1][0]
    pos_arbre.append(0)
  return locate_all_singleton_aux(next_tree,l,pos_arbre)

def get_max_singleton(arbre):
  l = locate_all_singleton(arbre)
  l.append((0,[]))
  print l
  h_max = max(l)
  return h_max[0]

def get_all_max_singleton(tree):
  l = locate_all_singleton(tree)
  maxi =-1
  liste_max = []
  for h,coord in l:
    if h>maxi:
      maxi=h
      liste_max = [(h,coord)]
    elif h==maxi:
      liste_max.append((h,coord))
  return liste_max

def split_generique(chaine,inter,motif):
  s,e = inter
  length = e - s
  patt = re.finditer(motif,chaine[s:e],re.UNICODE)
  l = []
  end = s
  for match in patt:
    start = match.start() + s
    end = match.end() + s
    couple = (start,end)
    l.append(couple)
  l.append((end,e))
  return l

def rec_split(s, i, list_seg, segmenter_i):
  start,end = i
  if segmenter_i == len(list_seg) :
    return [i, []]
  segmenter = list_seg[segmenter_i]
  l = segmenter(s, i)
  res = [i, []]
  for inter in l :
    sl = rec_split(s, inter, list_seg, segmenter_i+1)
    res[1].append(sl)
  return res

def parcours_prof2list(arbre, p, l):
  l.append(arbre)
  for fils in arbre[1]:
    parcours_prof2list(fils,p+1, l)








p = td.opt_parser_deft()
(o, _) = p.parse_args(sys.argv[1:])

list_file = []
if o.filein != "" :
  list_file = [o.filein]

else :
  d = os.path.join(o.dirin,'*')
  list_file = glob.glob(d)

list_file.sort()

for path in list_file :
  content = open(path).read()
  bs = BeautifulSoup(content)
  f = get_forest(bs)
#  nb_para,l,nb_resume = count_p(bs)
#  print path, nb_para, l, nb_resume



#print list_file
  

1/0


#content_unicode = unicode(content,'utf-8')

print content
1/0



bs = BeautifulSoup(content)
corps = bs.doc.article.corps
resume = bs.doc.article.resume
#print resume.prettify()
#print corps.prettify()

for i in corps :
  if type(i) == Tag :
    print i.string
  continue
  print "-"*30
  print type(i) == NavigableString
  print type(i)
  print "-"*30



1/0


def ouvrir_fichier(path):
  contenu = open(fichier,'r').read()
  return contenu

def split_par_html(s, i):
  motif = '(<p[^>]*>)(.*?)(</p>)'
  return split_triple_generique(s,i,motif)  

def split_par(s, i):
  motif = '([^\n]*)(\n{,1})'
  return split_generique(s,i,motif)

def split_sentence(s, i):
  motif = '([^\.]*)(\.{,1}[^\w]{1})'
  motif = '([^\.]*[\.^ ]*)(\. )'#tout sauf '. '
  motif = '( *)([\.\?!][\'"\)\]]* *)'
  return split_generique(s,i,motif)

def split_virgulot(s,i):
  motif = '([^,]*)(,{,1})'
  return split_generique(s,i,motif)

def split_word(s,i):
  motif = '([^ ]*)( {,1})'
  return split_generique(s,i,motif)

def split_triple_generique(chaine,intervalle,motif):
  s,e = intervalle
  patt = re.finditer(motif,chaine[s:e],re.DOTALL|re.UNICODE)
  l = []
  for match in patt:
    if match.group(2) != '' :
      start = match.start() + s + len(match.group(1))
      end = match.end() + s - len(match.group(3))
      couple = (start,end)
      l.append(couple)
  return l

def split_generique(chaine,intervalle,motif):
  s,e = intervalle
  patt = re.finditer(motif,chaine[s:e],re.UNICODE)#Empty matches included
  l = []
  for match in patt:
#    if start<end:# filtrage des ensembles vides
    if match.group(1) != '' :
      start = match.start() + s
      end = match.end() + s - len(match.group(2))
      couple = (start,end)
      l.append(couple)
#      print start, end
  return l

def rec_split(s, i, list_segmenters, segmenter_i):
  start,end = i
  if segmenter_i == len(list_segmenters) :
    return [i, []]
  segmenter = list_segmenters[segmenter_i]
  l = segmenter(s, i)
  res = [i, []]
  for inter in l :
    sl = rec_split(s, inter, list_segmenters, segmenter_i+1)
    res[1].append(sl)
  return res

def output_arbre(arbre,position):#parcours de l'arbre pour visualisation
#NB: Texte = arbre [0], paragraphes = arbre[1][1][x]....phrases, virgulots, mots
  hierarchie=['paragraphe','phrase','virgulot','mot']
  verbose =''
  compt=0
  for profondeur in position:
    verbose+=hierarchie[compt]+' '+str(profondeur)+', '
    try:
      arbre = arbre[1]#on descend d'un niveau
      arbre=arbre[profondeur]#on va chercher le n-ième élément
      compt+=1
    except:
      return False
  print verbose,arbre[0] 
  return arbre#Arbre[0] père, arbre[1] fils

def singlethon(arbre,chaine):#Donne la plus longue suite de singletons (départ,longueur)
  prec=(0,0)
  comp=0
  posit_arbre=[0]
  liste_sing={}
  for elem in Parcours_prof(arbre):
    noeud = elem[0]
    prof =  elem[1]
    if noeud==prec and prof==comp+1:#On descend mais les offsets sont inchangés
      print 'singleton',noeud
      print posit_arbre
    if prof>comp:#on est descendu d'un niveau
      posit_arbre.append(0)
    if prof<comp:#on est remonté
      for i in xrange(0,comp-prof):
        posit_arbre.pop()
      posit_arbre[prof]+=1
    if prof==comp:#on va en largeur
      posit_arbre[prof]+=1
    prec=noeud
    comp=prof
  
def parcours_prof2list(arbre, p, l):
#  l.append((arbre[0],p))
  l.append(arbre)
  for fils in arbre[1]:
    parcours_prof2list(fils,p+1, l)
    
class Parcours_prof:
    def __init__(self, tree):
        self._data = tree
    def __iter__(self):
        return self.next()
    def next(self):
      l = []
      parcours_prof2list(self._data, 0, l)
      for elt in l :
        yield elt

def parcours_larg(arbre):
  l = []
  for a in arbre[1]:
    l.append(a[0])
  return l


def hauteur(arbre):#hauteur = hauteur fils+1
  if len(arbre[1]) == 0 :
    return 1
  return 1 + max([hauteur(fils) for fils in arbre[1]])


class Parcours_sing:
    def __init__(self, tree):
        self._data = tree
    def __iter__(self):
        return self.next()
    def next(self):
      l = []
      rec_singleton(self._data, 0, l,[0])
      for elt in l :
        yield elt

def get_singleton_height(arbre):
#  print arbre,len(arbre)
  if len(arbre[1])!=1:
    return 0
  else:
    return 1+get_singleton_height(arbre[1][0])

def locate_all_singleton(arbre):
  return locate_all_singleton_aux(arbre,[],[]) 

def locate_all_singleton_aux(arbre,l,pos_arbre):
  h = get_singleton_height(arbre)
  if h == 0 :
    for i,son in enumerate(arbre[1]):
      pos_son = pos_arbre+[i]
      l = locate_all_singleton_aux(son,l,pos_son)
    return l
  l.append((h,pos_arbre[:]))
  next_tree = arbre
  for i in xrange(0,h):
    next_tree = next_tree[1][0]
    pos_arbre.append(0)
  return locate_all_singleton_aux(next_tree,l,pos_arbre) 




def locate_all_singleton_aux_bak(arbre,l,pos_arbre):
  if len(arbre[1])>1:
    for i,son in enumerate(arbre[1]):
      pos_son = pos_arbre+[i]
      l = locate_all_singleton_aux_bak(son,l,pos_son)
    return l
  elif len(arbre[1])==1:
    h = get_singleton_height(arbre)
    l.append((h,pos_arbre))
    next_tree = arbre
    pos_arbre = pos_arbre[:]

    for i in xrange(0,h):
      next_tree = next_tree[1][0]
      pos_arbre.append(0)

    return locate_all_singleton_aux_bak(next_tree,l,pos_arbre) 
  else:
    return l

def rec_singleton(arbre,f,l,pos_arbre):#f=suite singletons
  if arbre[1]:
    for fils in arbre[1]:
      nbr_fils=len(fils[1])
      freres=len(arbre[1])
      if freres>1:
        f=0
      if nbr_fils==1:
        f+=1
      else:
        if f>0:
          l.append((fils[0],f,pos_arbre))
        f=0 
      rec_singleton(fils,f,l,pos_arbre)

def get_max_singleton(arbre):
  l = locate_all_singleton(arbre)
  l.append((0,[]))
  h_max = max(l)#couple
  return h_max[0]

def get_all_max_singleton(tree):
  l = locate_all_singleton(tree)
  maxi =-1
  liste_max = []
  for h,coord in l:
    if h>maxi:
      maxi=h
      liste_max = [(h,coord)]
    elif h==maxi:
      liste_max.append((h,coord))
  return liste_max

def get_singleton_level(tree):#donne tous les max_sing pour un niveau de l'arbre
  out = []
  for son in tree[1]:
#    h = get_max_singleton(son)
    h = get_singleton_height(son)
    out.append(h)
  return out
def visual_tree(tree,chaine,i):
  for son in tree[1]:
    print '--'*i, chaine[son[0][0]:son[0][1]]
    visual_tree(son,chaine,i+1)

if __name__=="__main__":
  a_traiter = sys.argv [1:]
  liste_patt={}
  for fichier in a_traiter:
    contenu = ouvrir_fichier(fichier)
    seg_list = [split_word, split_virgulot, split_sentence, split_par_html]
    seg_list.reverse()
    intervalle_start= (0,len(contenu))
    r = rec_split(contenu, intervalle_start, seg_list, 0)
    for intervalle in r[1]:
      print '-'*10
      print contenu[intervalle[0][0]:intervalle[0][1]]
      print r[1][1]
      visual_tree(r,contenu,0)
    h = hauteur(r)
    l = get_singleton_level(r)
    l = tuple(l)
    if l not in liste_patt:
      liste_patt[l]=0
    liste_patt[l]+=1
  sort_patt=[(y,x) for x,y in liste_patt.iteritems()]
  sort_patt= sorted(sort_patt)
  for freq,patt in sort_patt:
    print freq,':',patt

#Agglutination des suites de 3, le reste= tête (premiers sous élém)+reste
#PLug in french_pipe  
